using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Text;
using System.Text.RegularExpressions;

namespace Atrax2s.Html
{
	public class TermExtractor
	{
		int m_wordCount = 0;
		public int WordCount { get { return m_wordCount; } set { m_wordCount = value; } }

		public TermExtractor()
		{
		}

		public Dictionary<string, double> ExtractX2Values(string content)
		{
			m_wordCount = 0;
			WordStripper ws = new WordStripper();
			List<string> rawparas = ws.StripToParagraphs(content, out m_wordCount);
			PrepareDenseParagraphSentences(rawparas);
			return Extract();
		}

		public Dictionary<string, double> ExtractX2Values(HtmlDocument doc)
		{
			m_wordCount = 0;
			string content = PrepareTextToDenseParagraphsUsingHtmlDoc(doc);
			return ExtractX2Values(content);
		}

		private Dictionary<string, double> Extract()
		{
			/*	The paragraphs KeyTerm collection is complete. Process extraction algorithm here.
			 * 
			 * The algorithm used here was found in a paper published by the
			 * International Journal of Artificial Intelligence Tools 
			 * Copyright 2003 (C) World Scientific Publishing Company
			 * Authored by Yutaka Matsuo of the National Institute of Advanced Industrial 
			 * Science and Technology in Tokyo, Japan and by Mitsuru Ishizuka of the University of Tokyo
			 * and published either in July or December of 2003 (unsure)
			 * 
			 * G = set of most frequent terms	  --> termsG = a subset of probabilityTerms where probability is above the average
			 * 
			 * Pg = sum of the total number of terms in sentences where             Dictionary<string, double> termPg
			 *      g appears divided by the total number of terms in the document
			 * 
			 * nw = total number of terms in sentences where w appears	 --> termNw
			 * 
			 * Fwg = sentence count where w and g occur divided by the total number of sentences
			 *       = termFwg is Dictionary<string, Dictionary<string, double>>
			 * 
			 * X2(w) is the rank for a give word w
			 * 
			 * X2(w) = sum of Z for each g in G (g = term in G or most frequent terms)
			 *         EXCEPT for the MAX g -- to create what the authors call robustness
			 * 
			 * D = nw * Pg
			 * 
			 * T = (Fwg - D)
			 * 
			 * Z = (T * T) / D
			 * 
			 * X2(w) = calculate Z for each g for w and sum the total
			 *         EXCEPT for the MAX g -- to create what the authors call robustness
			 * 
			*/

			//run through each sentence and grab two and three word segments and add them to the termCount
			this.AddMultiWordTerms();
			//LabLogger.Instance.Write("TermExtractor AddMultiWordTerms ran successfully.", 411, 01, LoggingCategory.All);

			this.SortTermsIntoProbabilities();	  //this gets us termsG for frequent terms, and an initialized termsX2
			//LabLogger.Instance.Write("TermExtractor SortTermsIntoProbabilities ran successfully.", 411, 01, LoggingCategory.All);

			this.FillTermPgNwCollections();   //now we have termPg and termNw filled with values
			//LabLogger.Instance.Write("TermExtractor FillTermPgNwCollections ran successfully.", 411, 01, LoggingCategory.All);

			this.FillTermFwgCollection();		//now we have the termFgw collection filled
			//LabLogger.Instance.Write("TermExtractor FillTermFwgCollection ran successfully.", 411, 01, LoggingCategory.All);

			string[] terms = new string[termsG.Count];
			termsG.Values.CopyTo(terms, 0);  //gives terms array where last term is the MAX g in G
			foreach (string w in terms)
			{
				double sumZ = 0;
				for (int i = 0; i < terms.Length - 1; i++) //do calcs for all but MAX
				{
					string g = terms[i];
					if (w != g) //skip where on the diagonal
					{
						int nw = termNw[w];
						double Pg = termPg[g];
						double D = nw * Pg;
						if (D != 0.0)
						{
							double Fwg = termFwg[w][terms[i]];
							double T = Fwg - D;
							double Z = (T * T) / D;
							sumZ += Z;
						}
					}
				}
				termsX2[w] = sumZ;
			}

			SortedDictionary<double, string> sortedX2 = new SortedDictionary<double, string>();
			foreach (KeyValuePair<string, double> pair in termsX2)
			{
				double x2 = pair.Value;
				while (sortedX2.ContainsKey(x2))
				{
					x2 = x2 - 0.0001;
				}
				sortedX2.Add(x2, pair.Key);
			}

			//now get simple array of values as lowest to highest X2 terms
			string[] x2Terms = new string[sortedX2.Count];
			sortedX2.Values.CopyTo(x2Terms, 0);

			Dictionary<string, double> result = new Dictionary<string, double>();
			for (int i = x2Terms.Length - 1; i > -1; i--)
			{
				string stemterm = x2Terms[i];
				string term = GetTermFromStemTerm(stemterm);
				if (term.Length > 0) result.Add(term, termsX2[x2Terms[i]]);
			}
			return result;
		}

		private string GetTermFromStemTerm(string term)
		{
			if (term.IndexOf(" ") > -1)
			{
				string[] terms = term.Split(' ');
				string[] words = new string[terms.Length];
				for (int i = 0; i < terms.Length; i++)
				{
					words[i] = GetTermFromStem(terms[i]);
				}
				string retval = string.Join(" ", words);
				return retval;
			}
			else
			{
				return GetTermFromStem(term);
			}
		}

		private string GetTermFromStem(string stem)
		{
			if (stems.ContainsKey(stem))
			{
				Dictionary<string, int> words = stems[stem];
				string word = string.Empty;
				int count = 0;
				foreach (KeyValuePair<string, int> pair in words)
				{
					if (pair.Value > count)
					{
						word = pair.Key;
						count = pair.Value;
					}
				}
				return word;
			}
			else
				return string.Empty;
		}

		private void FillTermFwgCollection()
		{
			//termFwg
			// * Fwg = sentence count where w and g occur divided by the total number of sentences (sentenceCount)
			// *       = termFwg is Dictionary<string, Dictionary<string, double>>

			string[] terms = new string[termsG.Count];
			termsG.Values.CopyTo(terms, 0);
			foreach (string w in terms)
			{
				foreach (KeyValuePair<double, string> pair in termsG)
				{
					string g = pair.Value;
					if (g != w)
					{
						int sentCountWG = 0;
						foreach (List<List<KeyTerm>> paragraph in paragraphs)
						{
							foreach (List<KeyTerm> sentence in paragraph)
							{
								if (this.TermsCoOccur(sentence, w, g)) sentCountWG++;
							}
						}
						double Fwg = sentCountWG > 0 ? sentCountWG / (double)sentenceCount : 0.0;
						if (!termFwg.ContainsKey(w))
							termFwg.Add(w, new Dictionary<string, double>()); //add if not there yet
						termFwg[w].Add(g, Fwg);
					}
				}
			}
		}

		private bool TermsCoOccur(List<KeyTerm> sentence, string w, string g)
		{
			if (TermInSentence(sentence, w) && TermInSentence(sentence, g))
				return true;
			else
				return false;
		}

		private bool TermInSentence(List<KeyTerm> sentence, string term)
		{
			bool found = false;
			//if term appears in this sentence, count the terms (words + 2 and 3 word terms)
			if (term.IndexOf(" ") > -1)
			{
				string[] termWords = term.Split(' ');
				for (int i = 0; i < sentence.Count; i++)
				{
					KeyTerm t = sentence[i];
					if (termWords.Length == 2 && i > 2)
					{
						KeyTerm t1 = sentence[i - 1];
						if (termWords[0] == t1.Stem && termWords[1] == t.Stem)
						{
							found = true;
							break;
						}
					}
					else if (termWords.Length == 3 && i > 3)
					{
						KeyTerm t1 = sentence[i - 1];
						KeyTerm t2 = sentence[i - 2];
						if (termWords[0] == t2.Stem && termWords[1] == t1.Stem && termWords[2] == t.Stem)
						{
							found = true;
							break;
						}
					}
				}
			}
			else
			{
				for (int i = 0; i < sentence.Count; i++)
				{
					KeyTerm t = sentence[i];
					if (t.Stem == term)
					{
						found = true;
						break;
					}
				}
			}
			return found;
		}

		private void FillTermPgNwCollections()
		{
			//termPg
			// * Pg = sum of the total number of terms in sentences where             Dictionary<string, double> termPg
			// *      g appears divided by the total number of terms in the document (termTotal)
			// total number of terms in sentence = word count + # of 2 and 3 word combos = termsInSentencesForTerm

			foreach (KeyValuePair<double, string> pair in termsG)
			{
				string term = pair.Value;
				int termsInSentencesForTerm = 0;
				foreach (List<List<KeyTerm>> paragraph in paragraphs)
				{
					foreach (List<KeyTerm> sentence in paragraph)
					{
						bool found = false;
						//if term appears in this sentence, count the terms (words + 2 and 3 word terms)
						if (term.IndexOf(" ") > -1)
						{
							string[] termWords = term.Split(' ');
							for (int i = 0; i < sentence.Count; i++)
							{
								KeyTerm t = sentence[i];
								if (termWords.Length == 2 && i > 2)
								{
									KeyTerm t1 = sentence[i - 1];
									if (termWords[0] == t1.Stem && termWords[1] == t.Stem)
									{
										found = true;
										break;
									}
								}
								else if (termWords.Length == 3 && i > 3)
								{
									KeyTerm t1 = sentence[i - 1];
									KeyTerm t2 = sentence[i - 2];
									if (termWords[0] == t2.Stem && termWords[1] == t1.Stem && termWords[2] == t.Stem)
									{
										found = true;
										break;
									}
								}
							}
						}
						else
						{
							for (int i = 0; i < sentence.Count; i++)
							{
								KeyTerm t = sentence[i];
								if (t.Stem == term)
								{
									found = true;
									break;
								}
							}
						}
						if (found)
						{
							//now get terms count (words + 2 and 3 word terms) and increment termsInSentencesForTerm
							termsInSentencesForTerm += sentence.Count;
							if(sentence.Count > 2) termsInSentencesForTerm += sentence.Count - 2; //all three word terms
							if (sentence.Count > 1) termsInSentencesForTerm += sentence.Count - 1; //all two word terms
						}
					}
				}
				termNw.Add(term, termsInSentencesForTerm);
				double pg = termsInSentencesForTerm / (double)this.termTotal;
				termPg.Add(term, pg);
			} //end foreach in termsG
		}

		private void SortTermsIntoProbabilities()
		{
			foreach (KeyValuePair<string, int> pair in termOccurrenceCounts)
			{
				termTotal += pair.Value;
			}
			double total = (double)termTotal;
			double probTotal = 0; //to be used for calculating the average probability
			foreach (KeyValuePair<string, int> pair in termOccurrenceCounts)
			{
				double prob = pair.Value / total;
				probTotal += prob;
				while (probabilityTerms.ContainsKey(prob))
				{
					prob = prob - 0.00113579; //offset by the slightest amount to get unique key
				}
				probabilityTerms.Add(prob, pair.Key);
			}
			double probAvg = termOccurrenceCounts.Count > 0 ? probTotal / termOccurrenceCounts.Count : 0;

			//only take the top 10% up to the top 30 terms and if top 10% is less than 10 then take up to 5
			int toptenCount = termOccurrenceCounts.Count / 10;
			if (toptenCount > 30)
				toptenCount = 30;
			else if (toptenCount < 10)
				toptenCount = 5;

			if (toptenCount > termOccurrenceCounts.Count) toptenCount = termOccurrenceCounts.Count; //just in case there are so few

			double[] ptkey = new double[probabilityTerms.Count];
			probabilityTerms.Keys.CopyTo(ptkey, 0);

			for (int i = ptkey.Length - 1; i > ptkey.Length - toptenCount - 1; i--)
			{
				double key = ptkey[i];
				string val = probabilityTerms[key];
				termsG.Add(key, val);
				termsX2.Add(val, 0); //initializes the list for storing X2 calculation results to be sorted later
			}

			//foreach (KeyValuePair<double, string> pair in probabilityTerms)
			//{
			//   if (pair.Key >= probAvg)
			//   {
			//      termsG.Add(pair.Key, pair.Value);
			//      termsX2.Add(pair.Value, 0); //initializes the list for storing X2 calculation results to be sorted later
			//   }
			//}
		}

		private void AddMultiWordTerms()
		{
			foreach (List<List<KeyTerm>> paragraph in paragraphs)
			{
				foreach (List<KeyTerm> sentence in paragraph)
				{
					for (int i = 0; i < sentence.Count; i++)
					{
						KeyTerm t = sentence[i];
						if (i > 0) //we can have a two word phrase
						{
							KeyTerm tm1 = sentence[i - 1];
							string term = tm1.Stem + " " + t.Stem;
							this.AddTerm(term);
						}
						if (i > 1) //we can have a three word phras
						{
							KeyTerm tm1 = sentence[i - 1];
							KeyTerm tm2 = sentence[i - 2];
							string term = tm2.Stem + " " + tm1.Stem + " " + t.Stem;
							this.AddTerm(term);
						}
					}
				}
			}
		}

		private void AddTerm(string stem)
		{
			if (termOccurrenceCounts.ContainsKey(stem))
				termOccurrenceCounts[stem]++;
			else
				termOccurrenceCounts.Add(stem, 1);
		}

		public List<List<List<KeyTerm>>> Paragraphs { get { return paragraphs; } }
		public Dictionary<string, Dictionary<string, int>> Stems { get { return stems; } }
		public Dictionary<string, int> TermCount { get { return termOccurrenceCounts; } }


		private void PrepareDenseParagraphSentences(List<string> rawparas)
		{
			/*
			 * remove all non-essential words: pronouns, helper verbs (to be forms, propositions, a, an, the, conjunctions
			 * 
			 * split text into sentences ( . ? ! ) and into words stemming each and adding
			 * to sentences, stems, and termCount (total occurence)
			*/
			//LabLogger.Instance.Write("TermExtractor PrepareTextToDenseParagraphs ran successfully with rawparas = " + rawparas.Count.ToString(), 411, 01, LoggingCategory.All);
			foreach (string rawpara in rawparas)
			{
				if (rawpara.Trim(trim).Length > 2) //ignore empty paragraphs
				{
					List<List<KeyTerm>> sentlist = new List<List<KeyTerm>>();
					string[] sents = rawpara.Split(endsent);
					foreach (string s in sents)
					{
						if (s.Trim(trim).Length > 2)
						{
							List<KeyTerm> words = new List<KeyTerm>();
							MatchCollection mc = wordreg.Matches(s);
							foreach (Match m in mc)
							{
								string word = m.Value.Trim(trim);
								if (word.Length > 2)	 //all two and one letter words are ignored
								{
									string stem = stemmer.Porter.stemTerm(word);
									KeyTerm term = new KeyTerm(word, stem);
									if (stems.ContainsKey(stem))
									{
										if (stems[stem].ContainsKey(word))
											stems[stem][word]++;
										else
											stems[stem].Add(word, 1);
									}
									else
									{
										stems.Add(stem, new Dictionary<string, int>());
										stems[stem].Add(word, 1);
									}
									this.AddTerm(stem);
									words.Add(term);
								}
							}
							if (words.Count > 0) //only add if we have words in the sentence
							{
								sentlist.Add(words);
								sentenceCount++; //increment total sentence count to be used later
							}
						}
					}
					if (sentlist.Count > 0) //only add paragraph if there are sentences
					{
						paragraphs.Add(sentlist);
					}
				}
			}
		}

		private string PrepareTextToDenseParagraphsUsingHtmlDoc(HtmlDocument doc)
		{
			StringBuilder sb = new StringBuilder();
			Collection<HtmlTag> pcdata = doc.GetList(DtdElement.PCDATA);
			//LabLogger.Instance.Write("TermExtractor doc.GetList(DtdElement.PCDATA) ran successfully with pcdata = " + pcdata.Count.ToString(), 411, 01, LoggingCategory.All);
			foreach (HtmlTag tag in pcdata)
			{
				HtmlTag prevtag = doc.Tags.Previous(tag);
				if(prevtag != null 
					&& (prevtag.Element == DtdElement.P 
						|| prevtag.Element == DtdElement.DIV 
						|| prevtag.Element == DtdElement.PRE 
						|| prevtag.Element == DtdElement.TD 
						|| prevtag.Element == DtdElement.BODY 
						|| prevtag.Element == DtdElement.BLOCKQUOTE
						|| prevtag.Element == DtdElement.BR 
						|| prevtag.Element == DtdElement.H1 
						|| prevtag.Element == DtdElement.H2 
						|| prevtag.Element == DtdElement.H3 
						|| prevtag.Element == DtdElement.H4 
						|| prevtag.Element == DtdElement.H5 
						|| prevtag.Element == DtdElement.H6 
						|| prevtag.Element == DtdElement.LI 
						|| prevtag.Element == DtdElement.TEXTAREA 
						|| prevtag.Element == DtdElement.TITLE ))
				{
					sb.Append("\r\n\r\n "); //forces a paragraph break when stripped
				}

				if (prevtag != null && prevtag.Element != DtdElement.NOSCRIPT) //don't count contain in the NOSCRIPT tag
					sb.Append(" " + doc.ReadSlice(tag.Slice).ToLower());

				HtmlTag nexttag = doc.Tags.Next(tag);
				if(nexttag != null)
				{
					DtdElement next = nexttag.Element;
					if (next == DtdElement.H1
						|| next == DtdElement.H2
						|| next == DtdElement.H3
						|| next == DtdElement.H4
						|| next == DtdElement.H5
						|| next == DtdElement.H6
						|| next == DtdElement.TITLE
						)
					{
						sb.Append(". "); //tag as sentence
					}
				}
			}
			return System.Web.HttpUtility.HtmlDecode(sb.ToString());
		}

		private Dictionary<string, Dictionary<string, double>> termFwg = new Dictionary<string, Dictionary<string, double>>();
		private Dictionary<string, int> termNw = new Dictionary<string, int>();
		private Dictionary<string, double> termPg = new Dictionary<string, double>();
		private Dictionary<string, double> termsX2 = new Dictionary<string, double>();
		private SortedDictionary<double, string> termsG = new SortedDictionary<double, string>();
		private SortedDictionary<double, string> probabilityTerms = new SortedDictionary<double, string>();
		private Stemmer stemmer = new Stemmer();
		private List<List<List<KeyTerm>>> paragraphs = new List<List<List<KeyTerm>>>();
		private Dictionary<string, Dictionary<string, int>> stems = new Dictionary<string, Dictionary<string, int>>(); //stem, words stemmed to it and the count
		private Dictionary<string, int> termOccurrenceCounts = new Dictionary<string, int>();
		private int termTotal = 0;
		//private HtmlDocument doc;
		private int sentenceCount = 0;
		private static Regex wordreg = new Regex(@"\b\S+?\b", RegexOptions.Compiled);
		private static char[] endsent = new char[3] { '.', '!', '?' };
		private static char[] trim = new char[5] { (char)247, ' ', '\t', '\n', '\r' };
	}

	public struct KeyTerm
	{
		public string Term;
		public string Stem;
		public KeyTerm(string term, string stem)
		{
			this.Term = term;
			this.Stem = stem;
		}
	}

	public class WordStripper
	{
		private static char[] p = new char[1] { (char)247 };
		private static string ps = null;
		public List<string> StripToParagraphs(string text, out int wordCount)
		{
			wordCount = 0;
			ps = new string(p);
			//LabLogger.Instance.Write("StripToParagraphs text = " + text, 411, 01, LoggingCategory.All);
			List<string> retval = new List<string>();
			try
			{
				MatchCollection mc = RegWordCount.Matches(text);
				wordCount = mc.Count;
				text = para.Replace(text, ps); //replace double CRLF with a acsii 01 for later splitting
				text = crlftab.Replace(text, " "); //replace remaining line breaks with simple space
				//remove all right ' for finding it's 
				text = rsquote.Replace(text, "'");
				//pronouns, helper verbs (to be forms, prepositions, a, an, the, conjunctions

				//LabLogger.Instance.Write("StripToParagraphs nonwords called text = " + text, 411, 01, LoggingCategory.All);
				//text = nonwords.Replace(text, "");
				text = non1.Replace(text, "");
				text = non2.Replace(text, "");
				text = non3.Replace(text, "");
				text = non4.Replace(text, "");
				text = non5.Replace(text, "");
				text = non6.Replace(text, "");
				text = non7.Replace(text, "");
				text = non8.Replace(text, "");
				text = non9.Replace(text, "");
				text = non10.Replace(text, "");
				text = non11.Replace(text, "");
				text = non12.Replace(text, "");
				text = non13.Replace(text, "");
				text = non14.Replace(text, "");
				text = non15.Replace(text, "");
				text = non16.Replace(text, "");
				text = non17.Replace(text, "");
				text = non18.Replace(text, "");
				text = non19.Replace(text, "");
				//remove large pockets of whitespace and replace with single space
				//LabLogger.Instance.Write("StripToParagraphs white called text = " + text, 411, 01, LoggingCategory.All);
				text = white.Replace(text, " ");

				//LabLogger.Instance.Write("StripToParagraphs split called text = " + text, 411, 01, LoggingCategory.All);
				string[] paras = text.Split(p);

				retval = new List<string>(paras);
			}
			catch(Exception e)
			{
				//Log.App(LogEvents.WordStripperStripToParagraphsFailed, e);
				System.Diagnostics.Debug.WriteLine(e.ToString());
			}
			return retval;
		}
		public static Regex RegWordCount = new Regex(@"\b\S+?\b", RegexOptions.Compiled);
		static Regex para = new Regex("\r\n\r\n", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex crlftab = new Regex("\r\n|\t", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex lsquote = new Regex(@"\u2018", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex rsquote = new Regex(@"\u2019", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex ldquote = new Regex(@"\u201C", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex rdquote = new Regex(@"\u201D", (RegexOptions.IgnoreCase | RegexOptions.Compiled));


		static Regex non1 = new Regex(@"\b(a|aboard|about|above|absent|according\sto|across|after|against|ago|ahead\sof|ain't|all|along|alongside)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non2 = new Regex(@"\b(also|although|am|amid|amidst|among|amongst|an|and|anti|anybody|anyone|anything|apart|apart\sfrom|are|been)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non3 = new Regex(@"\b(aren't|around|as|as\sfar\sas|as\ssoon\sas|as\swell\sas|aside|at|atop|away|be|because|because\sof|before)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non4 = new Regex(@"\b(behind|below|beneath|beside|besides|between|betwixt|beyond|but|by|by\smeans\sof|by\sthe\stime|can|cannot)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non5 = new Regex(@"\b(circa|close\sto|com|concerning|considering|could|couldn't|cum|'d|despite|did|didn't|do|does|doesn't|don't)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non6 = new Regex(@"\b(down|due\sto|during|each_other|'em|even\sif|even\sthough|ever|every|every\stime|everybody|everyone)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non7 = new Regex(@"\b(everything|except|far\sfrom|few|first\stime|following|for|from|get|got|had|hadn't|has|hasn't|have)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non8 = new Regex(@"\b(haven't|he|hence|her|here|hers|herself|him|himself|his|how|i|if|in|in\saccordance\swith|in\saddition\sto|in\scase)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non9 = new Regex(@"\b(in\sfront\sof|in\slieu\sof|in\splace\sof|in\sspite\sof|in\sthe\sevent\sthat|in\sto|inside|inside\sof)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non10 = new Regex(@"\b(instead\sof|into|is|isn't|it|itself|just\sin\scase|like|'ll|lots|may|me|mid|might|mightn't|mine|more|most)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non11 = new Regex(@"\b(must|mustn't|myself|near|near\sto|nearest|no|no\sone|nobody|none|not|nothing|notwithstanding|now\sthat|of)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non12 = new Regex(@"\b(off|on|on\sbehalf\sof|on\sto|on\stop\sof|once|one|one\sanother|only\sif|onto|opposite|or|org|other|our)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non13 = new Regex(@"\b(ours|ourselves|out|out\sof|outside|outside\sof|over|past|per|plenty|plus|prior\sto|qua|re|'re|really)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non14 = new Regex(@"\b(regarding|round|'s|said|sans|save|say|says|shall|shan't|she|should|shouldn't|since|so|somebody)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non15 = new Regex(@"\b(someone|something|than|that|the|thee|their|theirs|them|themselves|there|these|they|thine|this|thou)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non16 = new Regex(@"\b(though|through|throughout|till|to|toward|towards|under|underneath|unless|unlike|until|unto|up|upon)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non17 = new Regex(@"\b(us|'ve|versus|via|was|wasn't|we|were|weren't|what|when|whenever|where|whereas|whether\sor\snot)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non18 = new Regex(@"\b(while|who|whoever|whom|why|will|with|with\sregard\sto|withal|within|without|won't|would|wouldn't)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		static Regex non19 = new Regex(@"\b(ya|ye|yes|you|your|yours|yourself)\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		

		//replaced with multiples -- had a problem in x64 server with race condition using this large regex
		//private static Regex nonwords = new Regex(@"\b("
		//   + @"a|aboard|about|above|absent|according\sto|across|after|against|ago|ahead\sof|ain't|all|along|alongside|"
		//   + @"also|although|am|amid|amidst|among|amongst|an|and|anti|anybody|anyone|anything|apart|apart\sfrom|are|"
		//   + @"aren't|around|as|as\sfar\sas|as\ssoon\sas|as\swell\sas|aside|at|atop|away|be|because|because\sof|before|"
		//   + @"behind|below|beneath|beside|besides|between|betwixt|beyond|but|by|by\smeans\sof|by\sthe\stime|can|cannot|"
		//   + @"circa|close\sto|com|concerning|considering|could|couldn't|cum|'d|despite|did|didn't|do|does|doesn't|don't|"
		//   + @"down|due\sto|during|each_other|'em|even\sif|even\sthough|ever|every|every\stime|everybody|everyone|"
		//   + @"everything|except|far\sfrom|few|first\stime|following|for|from|get|got|had|hadn't|has|hasn't|have|"
		//   + @"haven't|he|hence|her|here|hers|herself|him|himself|his|how|i|if|in|in\saccordance\swith|in\saddition\sto|in\scase|"
		//   + @"in\sfront\sof|in\slieu\sof|in\splace\sof|in\sspite\sof|in\sthe\sevent\sthat|in\sto|inside|inside\sof|"
		//   + @"instead\sof|into|is|isn't|it|itself|just\sin\scase|like|'ll|lots|may|me|mid|might|mightn't|mine|more|most|"
		//   + @"must|mustn't|myself|near|near\sto|nearest|no|no\sone|nobody|none|not|nothing|notwithstanding|now\sthat|of|"
		//   + @"off|on|on\sbehalf\sof|on\sto|on\stop\sof|once|one|one\sanother|only\sif|onto|opposite|or|org|other|our|"
		//   + @"ours|ourselves|out|out\sof|outside|outside\sof|over|past|per|plenty|plus|prior\sto|qua|re|'re|really|"
		//   + @"regarding|round|'s|said|sans|save|say|says|shall|shan't|she|should|shouldn't|since|so|somebody|"
		//   + @"someone|something|than|that|the|thee|their|theirs|them|themselves|there|these|they|thine|this|thou|"
		//   + @"though|through|throughout|till|to|toward|towards|under|underneath|unless|unlike|until|unto|up|upon|"
		//   + @"us|'ve|versus|via|was|wasn't|we|were|weren't|what|when|whenever|where|whereas|whether\sor\snot|"
		//   + @"while|who|whoever|whom|why|will|with|with\sregard\sto|withal|within|without|won't|would|wouldn't|"
		//   + @"ya|ye|yes|you|your|yours|yourself"
		//   + @")\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));

		//removed dupes and alphabatized list
		//private static Regex nonwords = new Regex(@"\b("
		//   + @"it|i|you|he|they|she|we|who|them|him|me|her|one|us|something|nothing|himself|"
		//   + @"anything|itself|themselves|someone|everything|herself|anyone|everyone|whom|myself|each_other|yourself|none|no_one|"
		//   + @"somebody|nobody|everybody|anybody|his|plenty|mine|lots|ourselves|yours|one\sanother|hers|no\sone|ours|'em|these|"
		//   + @"ye|whoever|ya|their|theirs|thee|thou|thine|our|other|"
		//   + @"am|aren't|are|aren't|ain't|'re|be|can|cannot|could|couldn't|did|didn't|do|don't|does|"
		//   + @"doesn't|had|hadn't|'d|has|hasn't|'s|have|haven't|'ve|is|isn't|'s|may|might|mightn't|must|mustn't|shall|shan't|should|"
		//   + @"shouldn't|was|wasn't|were|weren't|will|won't|'ll|would|wouldn't|'d|"
		//   + @"as\sfar\sas|as\swell\sas|by\smeans\sof|in\saccordance\swith|in\slieu\sof|in\saddition\sto|"
		//   + @"in\sfront\sof|in\splace\sof|in\sspite\sof|on\sbehalf\sof|on\stop\sof|with\sregard\sto|according\sto|ahead\sof|apart\sfrom|"
		//   + @"because\sof|close\sto|due\sto|far\sfrom|in\sto|inside\sof|instead\sof|on\sto|out\sof|outside\sof|near\sto|prior\sto|aboard|"
		//   + @"withal|about|above|absent|across|after|against|along|alongside|amid|amidst|among|amongst|around|as|at|atop|before|behind|"
		//   + @"below|beneath|beside|besides|between|beyond|by|despite|down|during|following|for|from|in|inside|into|like|mid|near|nearest|"
		//   + @"notwithstanding|of|off|on|onto|opposite|out|outside|over|past|re|round|since|through|throughout|till|to|toward|towards|"
		//   + @"under|underneath|unlike|until|up|upon|via|with|within|without|anti|betwixt|circa|cum|per|qua|sans|unto|versus|concerning|"
		//   + @"considering|regarding|but|except|plus|save|ago|apart|aside|away|hence|through|"
		//   + @"a|an|the|"
		//   + @"after|although|as\ssoon\sas|as|because|before|by\sthe\stime|even\sif|even\sthough|every\stime|if|in\scase|in\sthe\sevent\sthat|"
		//   + @"just\sin\scase|now\sthat|once|only\sif|since|first\stime|though|unless|until|when|whenever|whereas|whether\sor\snot|"
		//   + @"while|and|but|or|"
		//   + @"more|than|so|not|that|all|also|this|not|no|yes|your|really|here|ever|so|say|says|said|there|why|where|what|how|"
		//   + @"every|few|most|get|got|com|org"
		//   + @")\b", (RegexOptions.IgnoreCase | RegexOptions.Compiled));
		private static Regex white = new Regex(@"\s+", RegexOptions.Compiled);
	}
}


